package com.openness.example.crawler;

import java.util.regex.Pattern;

import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;

import com.openness.crawler.crawler.Page;
import com.openness.crawler.crawler.WebCrawler;
import com.openness.crawler.hbase.HBaseTablePool;
import com.openness.crawler.url.WebURL;

/**
 * @author Rain
 * @version 2013-5-20 下午2:56:52
 */

public class MyCrawler extends WebCrawler {

	private final static Pattern FILTERS = Pattern
			.compile(".*(\\.(css|js|bmp|gif|jpe?g"
					+ "|png|tiff?|mid|mp2|mp3|mp4"
					+ "|wav|avi|mov|mpeg|ram|m4v|pdf"
					+ "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

	private final static Pattern MATCHES = Pattern
			.compile("^(http|https)+{1}(://){1}(.)+((\\.html)|(\\.shtml)|(\\.htm)|(\\.htm)|(\\.asp)|(\\.jsp)|(\\.php)|(\\.cgi)|(\\.perl)){1}(.)*$");

	/**
	 * You should implement this function to specify whether the given url
	 * should be crawled or not (based on your crawling logic).
	 */
	@Override
	public boolean shouldVisit(WebURL url) {
		Statistic.addAllUrl();

		String href = url.getURL().toLowerCase();

		boolean visit = false;

		if (!FILTERS.matcher(href).matches() && MATCHES.matcher(href).matches()) {
			visit = true;

			Statistic.addCrawlerUrl();
		}

		//Statistic.stat();

		return visit;
	}

	/**
	 * This function is called when a page is fetched and ready to be processed
	 * by your program.
	 */
	@Override
	public void visit(Page page) {
		if (page.getContentData() != null && page.getContentData().length > 0) {
			Statistic.addCrawlerPageSuccess();
		}

		HTableInterface table = null;

		try {
			table = HBaseTablePool.get("mycrawler");

			Put put = new Put(Bytes.toBytes(page.getWebURL().getDocid()));

			put.add(Bytes.toBytes("baseinfo"), Bytes.toBytes("url"),
					Bytes.toBytes(page.getWebURL().getURL()));

			table.put(put);
		} catch (Exception e) {
			System.out.println("MyCrawler visit error: " + e.toString());
		} finally {
			HBaseTablePool.put(table);
		}
		
		Statistic.addCrawlerDataSuccess();
	}

}
